#coding=utf-8

import urllib, urllib2
import re
import time
import hashlib
import random
import os,sys
from config import *
from pycode.commfunc import * 


def getUrlData(url):
    data = ''
    try:
        data = urllib.urlopen(url).read()
    except IOError, e:
        if str(e).find('[Errno url error] unknown url type: \'e\'') > -1:
            url = "file://%s"%url.replace("\\",'/').replace(":",'|')
            data = urllib.urlopen(url).read()
        elif str(e).find('[Errno socket error] [Errno') > -1:
            for i in xrange(5):
                try:
                    data = urllib.urlopen(url).read()
                except:
                    pass
                if data:
                    break
        else:
            raise e
    return data

def getExtName(fileName):
    '''
    获取文件后缀名
    '''
    #反转
    reName = fileName[::-1]
    nb = reName.find('.')
    newName = ''
    if nb != -1:
        newName = reName[:nb][::-1]
    return newName

def downImg(name, listid, catalogid , path):
    returnData = {
        'path':'',
        'thumbpath':0
    } #返回值

    #为了以后seo，把图片的格式、系列名字保存为文件名，最后加上随机数
    #先用python下载图片后再用php获取图片的信息及生成缩略图
    #name= 'abcde'
    #下载
    addPath = "./pics/%s/%s/"%(listid, catalogid)
    if False == os.path.isdir(addPath):
        os.makedirs(addPath)
    ext = getExtName(path)
    seek = "%s_%s"%(name, hashlib.md5(str(time.time()) + str(random.uniform(1,10))).hexdigest().upper())
    newPath = seek[:-20]+random.choice('abcdefghizf0123456789')
    retNewPath = newPath + '.' +ext #返回的是没有目录的图片地址
    newPathName = addPath + newPath + '.' + ext
    newPathNameThumb = addPath + newPath + '_thumb' + '.' + ext
    ret = False
    try:
        urllib.urlretrieve(path, newPathName) #生成图片
        print newPathName
        print newPathNameThumb
        ret = True
    except Exception, e:
        time.sleep(2)
        for i in range(5):
            try:
                urllib.urlretrieve(path, newPathName)
                ret = True
            except:
                pass
            if ret == True:
                break
        

    if ret == True:
        returnData['path'] = retNewPath
        tRet = False
        try:
            cmd = "php image.php %s %s"%(newPathName, newPathNameThumb)
            os.system(cmd) #生成缩略图
            tRet = True
        except Exception, e:
            time.sleep(2)
            for i in range(5):
                time.sleep(2)
                try:
                    cmd = "php image.php %s %s"%(newPathName, newPathNameThumb)
                    os.system(cmd)
                    tRet = Ture
                except:
                    pass
                if tRet == True:
                    break
        if tRet == True:
            returnData['thumbpath'] = 1
        return returnData
      
    else:
        return {}


def downItemData(limit = 10):
    db = sqldb
    sql = "SELECT id,url FROM catalog WHERE isfinish=0 or isfinish=9 order by id asc limit 200"
    rs = db.fetchall(sql)
    if rs:
        return rs



class Base(object):
    def __init__(self, url):
        self.url = url
        self.data = ''
        self.db = sqldb
        self.timestamp = int(time.time())

class HotCatalog(Base):
    def __init__(self,url):
        Base.__init__(self,url)
        self.listid = 1

    def getPageData(self, url=''):
        if url == '':
            self.data = getUrlData(self.url)       
        else:
            self.data = getUrlData(url)
        return self.data

    def getData(self, url=''):
        return self.getPageData(url)

    def getPageNumber(self):
        data = self.getData()
        m = re.findall('\/(\d+)<\/span>', data, re.M)
        if [] == m:
            return ''
        else:
            return int(m[0])

    def insertCatalog(self, url, title):
        sql = "SELECT id FROM catalog WHERE url='%s'"\
              %(url,)
        rs = self.db.fetchone(sql)
        if None == rs:
            sql = "INSERT INTO catalog (listid,url,name)VALUES('%s','%s','%s')"\
                  %(self.listid, url, addslashes(title))
            self.db.execute(sql)
            return self.db.lastId()
        else:
            return rs['id']
    
    def insertCatalogAll(self):
        pageNumber = self.getPageNumber()
        
        url = self.url[:-7]
        for i in range(1, pageNumber):
            
            turl = url + '-' + str(i) + '.html'
            
            self.insertCatalogList(turl)
    
    def insertCatalogList(self, url=''):
        data = self.getData(url)
        m = re.findall('\/span><a href="\/article\/html\/(.*)\.html"\W+target="_blank" title="(.*)"', data, re.M)
        if [] == m:
            return ''
        else:
            for item in m:
                url = 'http://51hot.net/article/html/%s.html'%item[0]
                self.insertCatalog(url, item[1])
'''
client = HotCatalog('http://51hot.net/article/html/list269-1.html')
client.insertCatalogAll()
#client.getListUrls()
exit()
'''
class HotItem(Base):
    def __init__(self, url):
        Base.__init__(self, url)
        self.listid = 1
        self.title = self.getTitle()
        self.catalogid = self.insertCatalog()
        self.isfinish = self.checkIsfinish()
        
    def checkIsfinish(self):
        sql = "SELECT isfinish FROM catalog WHERE id=%s"\
              %(self.catalogid,)
        res = self.db.fetchone(sql)
        if res['isfinish'] == 1:
            return 1
        else:
            return 0
            

    def insertCatalog(self):
        url = self.url
        title = self.title
        sql = "SELECT id FROM catalog WHERE url='%s'"\
              %(url,)
        rs = self.db.fetchone(sql)
        if None == rs:
            sql = "INSERT INTO catalog (listid,url,name)VALUES('%s','%s','%s')"\
                  %(self.listid, url, addslashes(title))
            self.db.execute(sql)
            return self.db.lastId()
        else:
            return rs['id']

    def insertItem(self, data):
        sql = "INSERT INTO item (catalogid,url,hasthumb,createtime) VALUES"
        vals = ''
        for item in data:
            if item == {}:
                continue
            self.db.execute(sql + "(%s,'%s','%s','%s');"%(self.catalogid, item['path'], item['thumbpath'], self.timestamp), 1)
        self.db.commit()

    
    def getPageContent(self):
        self.data = getUrlData(self.url)       
        return self.data

    def getData(self):
        return self.getPageContent()
    
    def getPageImg(self, pageNumber=''):
        if pageNumber == '':
            data = self.getData()
        else:
            url = "%s"%self.url[:-5] +'_' + str(pageNumber) + '.html'
            data = getUrlData(url)
        if data == '':
            return ''
        #print self.url
        
        partern = '<div class="mainNewsContent NewsContent" id="mainNewsContent">(.*|<br\/>|<br\/><br\/>|<br\/>\s+<br\/>\s+)\s+(.*|<p[^]+>)(.*|<a[^>]+>)(.*)(.*|\s+)<\/p><p'
        mm = re.findall(partern, data, re.M)
        mt = re.search('<img[^>]+src=[\'|\"](.*)[\'|\"]', mm[0][0])
        if mt == None:
            m = re.findall('<img[^>]+src=[\'|\"](.*\.jpg|\.gif|\.bmp|\.png)[\'|\"]', mm[0][1], re.M)

        else:
            m = re.findall('<img[^>]+src=[\'|\"](.*\.jpg|\.gif|\.bmp|\.png)[\'|\"]', mm[0][0], re.M)

        if [] == m:
            return ''
        else:
            return m[0]

    def getTitle(self):
        return self.getPageTitle()

    def getPageTitle(self):
        data = self.getData()
        m = re.findall('<H1>(.*)<\/H1>', data)
        if [] == m:
            return ''
        else:
            self.title = m[0]
            return self.title

    def getSeriesContentByPages(self):
        '''
        获取系列的所有数据，包括分页
        '''
        allImgPath = []
        data = self.getData()
        imgPath = self.getPageImg()
        if imgPath == '':
            return None
        if '' != imgPath:
            allImgPath.append(imgPath)
            imgPath = ''
        parent = '\[(\d+)\]<\/a>&nbsp;&nbsp;<a href="\d+_2\.html">(.*)<\/a>'
        m = re.findall(parent, data)
        if [] == m:
            pass
        else:
            pageNumber = int(m[0][0]) + 1
            for i in range(2, pageNumber):
                imgPath = self.getPageImg(i)
                if '' != imgPath:
                    allImgPath.append(imgPath) 
        return allImgPath
    
    def download(self):
        if self.isfinish == 1:
            print self.catalogid
            return
        title = self.getTitle()
        
        allImages = self.getSeriesContentByPages()
        if allImages == None:
            print 'go out'
            print self.url
            self.updateCatalog(9) #9 为有问题，暂时采集不了的数据
            return None
        print title
        print self.url
        urls = []
        for i in allImages:
            ipt = "http://51hot.net" + i
            downRet = downImg(title, self.listid, self.catalogid, ipt)
            if '' != downRet:
                urls.append(downRet)
        self.insertItem(urls)
        self.updateCatalog(1)

    def updateCatalog(self, fg):
        sql = "UPDATE catalog SET isfinish=%s WHERE id=%s"\
              %(fg, self.catalogid)
        self.db.execute(sql)
url = 'http://51hot.net/article/html/84506.html'

urls = downItemData()
for item in urls:
    client = HotItem(item['url'])
    client.download()